import asyncio
import os
import json
import logging
from pymongo import MongoClient, errors
import config
from datetime import datetime

# 配置日志记录
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s',
    handlers=[
        logging.StreamHandler(),
        logging.FileHandler('crawler.log')
    ]
)

def get_platform_folder_name(platform):
    """将平台缩写映射到对应的文件夹名称"""
    platform_mapping = {
        'dy': 'douyin',
        'xhs': 'xiaohongshu',
        'bili': 'bilibili',
        'ks': 'kuaishou',
        'wb': 'weibo',
        'tieba': 'tieba',
        'zhihu': 'zhihu'
    }
    return platform_mapping.get(platform, platform)

def get_collection_name(data_type):
    """生成collection名称"""
    current_time = datetime.now().strftime('%Y%m%d_%H%M%S')
    # 处理关键词，移除特殊字符
    keywords = ''.join(char for char in config.KEYWORDS if char.isalnum() or char in ['_', '-'])
    collection_name = f"{config.PLATFORM}_{keywords}_{data_type}_{current_time}"
    return collection_name

async def read_json_file(file_path):
    """异步读取JSON文件"""
    loop = asyncio.get_event_loop()
    try:
        with open(file_path, 'r', encoding='utf-8') as file:
            return await loop.run_in_executor(None, json.load, file)
    except json.JSONDecodeError as e:
        logging.error(f"JSON解析失败: {e}")
        return None
    except Exception as e:
        logging.error(f"读取文件失败: {e}")
        return None

async def save_data_to_mongodb(data, collection):
    """将数据保存到MongoDB"""
    if not data:
        logging.warning("无有效数据可保存")
        return False
    
    try:
        if isinstance(data, list):
            if len(data) == 0:
                logging.warning("空列表数据，跳过保存")
                return False
            result = collection.insert_many(data)
            logging.info(f"成功插入 {len(result.inserted_ids)} 条文档")
        else:
            result = collection.insert_one(data)
            logging.info("成功插入1条文档")
        return True
    except errors.PyMongoError as e:
        logging.error(f"MongoDB操作失败: {e}")
        return False

async def create_event_in_public_sentiment(client, platform, keywords, comments_count, contents_count):
    """在public_sentiment数据库中创建事件 如果成功则返回id"""
    try:
        db = client['public_sentiment']
        events_collection = db['events']
        
        # 获取当前最大的id值并加1
        max_id_doc = events_collection.find_one(sort=[("id", -1)])
        new_id = max_id_doc['id'] + 1 if max_id_doc else 1
        
        # 清理关键词
        clean_keywords = ''.join(char for char in keywords if char.isalnum() or char in ['_', '-'])
        
        # 创建事件文档（按要求将type、sentiment、region置空）
        event_doc = {
            "id": new_id,
            "eventstate":0,#0-刚爬到未处理的事件，1-可更新的事件,2-已处理的事件,-1-处理中（仅front中用到）
            "title": clean_keywords,
            "date": datetime.utcnow(),
            "type": None,          # 置空
            "platform": platform,
            "sentiment": None,     # 置空
            "comments": comments_count,
            "region": "全国",        # 置空
            "articles": contents_count,
            "keywords": [clean_keywords],
            "description": f"{clean_keywords}相关事件",
            "hotwords": [],
            "genderData": [0,0],
            "sentimentData": [0,0,0] 
        }
        
        # 插入事件
        result = events_collection.insert_one(event_doc)
        if result.inserted_id:
            logging.info(f"成功创建事件: {clean_keywords} (ID: {new_id})")
            return new_id
        return False
    except Exception as e:
        logging.error(f"创建事件失败: {e}")
        return False

# async def save_to_mongodb():
#     """主函数：将JSON数据保存到MongoDB"""
#     # 获取当前文件的绝对路径
#     current_dir = os.path.dirname(os.path.abspath(__file__))
    
#     # 使用映射后的平台名称构建路径
#     platform_folder = get_platform_folder_name(config.PLATFORM)
#     folder_path = os.path.join(current_dir, 'data', platform_folder, 'json')
    
#     logging.info(f"数据目录: {folder_path}")
    
#     # 创建目录（如果不存在）
#     os.makedirs(folder_path, exist_ok=True)
    
#     # 连接MongoDB
#     try:
#         client = MongoClient(
#             'mongodb://localhost:27017/',
#             serverSelectionTimeoutMS=5000  # 5秒连接超时
#         )
#         # 测试连接
#         client.server_info()
#         db = client['crawler']
#     except errors.PyMongoError as e:
#         logging.error(f"MongoDB连接失败: {e}")
#         return

#     # 处理JSON文件
#     processed_files = 0
#     comments_count = 0
#     contents_count = 0
#     event_created = False
    
#     for filename in os.listdir(folder_path):
#         if not filename.endswith('.json'):
#             continue
            
#         file_path = os.path.join(folder_path, filename)
#         logging.info(f"正在处理文件: {filename}")
        
#         try:
#             data = await read_json_file(file_path)
#             if not data:
#                 continue
                
#             # 确定数据类型
#             if 'comments' in filename:
#                 data_type = 'comments'
#                 is_comments = True
#             elif 'contents' in filename:
#                 data_type = 'contents'
#                 is_comments = False
#             else:
#                 logging.warning(f"无法识别文件 {filename} 的数据类型，跳过")
#                 continue
                
#             # 获取集合名称并保存数据
#             collection_name = get_collection_name(data_type)
#             collection = db[collection_name]
            
#             if await save_data_to_mongodb(data, collection):
#                 processed_files += 1
#                 # 统计文档数量
#                 if is_comments:
#                     comments_count += len(data) if isinstance(data, list) else 1
#                 else:
#                     contents_count += len(data) if isinstance(data, list) else 1
                
#         except Exception as e:
#             logging.error(f"处理文件 {filename} 时出错: {e}")
#             continue
    
#     # 所有文件处理完成后，创建事件（只创建一次）
#     if processed_files > 0 and not event_created:
#         logging.info("尝试创建事件...")
#         if await create_event_in_public_sentiment(
#             client=client,
#             platform=config.PLATFORM,
#             keywords=config.KEYWORDS,
#             comments_count=comments_count,
#             contents_count=contents_count
#         ):
#             event_created = True
#             logging.info("事件创建成功")
#         else:
#             logging.warning("事件创建失败")

#     # 关闭连接
#     client.close()
#     logging.info(f"处理完成，共成功处理 {processed_files} 个文件")


async def save_to_mongodb():
    """主函数：将JSON数据保存到MongoDB"""
    # 获取当前文件的绝对路径
    current_dir = os.path.dirname(os.path.abspath(__file__))
    
    # 使用映射后的平台名称构建路径
    platform_folder = get_platform_folder_name(config.PLATFORM)
    folder_path = os.path.join(current_dir, 'data', platform_folder, 'json')
    
    logging.info(f"数据目录: {folder_path}")
    
    # 创建目录（如果不存在）
    os.makedirs(folder_path, exist_ok=True)
    
    # 连接MongoDB
    try:
        client = MongoClient(
            'mongodb://localhost:27017/',
            serverSelectionTimeoutMS=5000  # 5秒连接超时
        )
        # 测试连接
        client.server_info()
        db = client['crawler']
    except errors.PyMongoError as e:
        logging.error(f"MongoDB连接失败: {e}")
        return

    # 处理JSON文件
    processed_files = 0
    comments_count = 0
    contents_count = 0
    event_created = False
    comments_collection_name = None
    contents_collection_name = None
    
    for filename in os.listdir(folder_path):
        if not filename.endswith('.json'):
            continue
            
        file_path = os.path.join(folder_path, filename)
        logging.info(f"正在处理文件: {filename}")
        
        try:
            data = await read_json_file(file_path)
            if not data:
                continue
                
            # 确定数据类型
            if 'comments' in filename:
                data_type = 'comments'
                is_comments = True
                comments_collection_name = get_collection_name(data_type)
            elif 'contents' in filename:
                data_type = 'contents'
                is_comments = False
                contents_collection_name = get_collection_name(data_type)
            else:
                logging.warning(f"无法识别文件 {filename} 的数据类型，跳过")
                continue
                
            # 获取集合名称并保存数据
            collection_name = get_collection_name(data_type)
            collection = db[collection_name]
            
            if await save_data_to_mongodb(data, collection):
                processed_files += 1
                # 统计文档数量
                if is_comments:
                    comments_count += len(data) if isinstance(data, list) else 1
                else:
                    contents_count += len(data) if isinstance(data, list) else 1
                
        except Exception as e:
            logging.error(f"处理文件 {filename} 时出错: {e}")
            continue
    
    # 所有文件处理完成后，创建事件（只创建一次）
    event_id = None
    if processed_files > 0 and not event_created:
        logging.info("尝试创建事件...")
        event_id = await create_event_in_public_sentiment(
            client=client,
            platform=config.PLATFORM,
            keywords=config.KEYWORDS,
            comments_count=comments_count,
            contents_count=contents_count
        )
        if event_id:
            event_created = True
            logging.info(f"事件创建成功，ID: {event_id}")
            
            # 调用数据处理和情感分析函数
            if comments_collection_name and contents_collection_name:
                try:
                    logging.info("开始数据处理和情感分析...")
                    # 调用训练好的模型以及格式转换
                    # TODO 更改的存放位置（T.py Two_sentiment.py SentimentTwo）
                    
                    import T
                    T.data_transformA(comments_collection_name, contents_collection_name, event_id)
                    logging.info("数据处理和情感分析完成")
                except Exception as e:
                    logging.error(f"数据处理和情感分析失败: {e}")
        else:
            logging.warning("事件创建失败")

    # 关闭连接
    client.close()
    logging.info(f"处理完成，共成功处理 {processed_files} 个文件")

    # 删除文件夹中的所有文件
    try:
        for filename in os.listdir(folder_path):
            file_path = os.path.join(folder_path, filename)
            try:
                if os.path.isfile(file_path):
                    os.unlink(file_path)
                    logging.info(f"已删除文件: {file_path}")
                # 如果是子目录，可以添加处理逻辑
                # elif os.path.isdir(file_path): shutil.rmtree(file_path)
            except Exception as e:
                logging.error(f"删除文件 {file_path} 失败: {e}")
        logging.info(f"已清空文件夹: {folder_path}")
    except Exception as e:
        logging.error(f"清空文件夹时出错: {e}")

if __name__ == '__main__':
    try:
        asyncio.run(save_to_mongodb())
    except KeyboardInterrupt:
        logging.info("程序被用户中断")
    except Exception as e:
        logging.error(f"程序运行出错: {e}")